"""
主题: 简单的并行编程
问题: 你有个程序要执行CPU密集型工作，你想让他利用多核CPU的优势来运行的快一点。
提示 : 
"""


'''日志文件中查找出所有访问过robots.txt文件的主机'''

import gzip
import io
import glob # 主要是用来查找文件与目录的

def find_robots(filename):
    '''
    Find all of the hosts that access robots.txt in a single log file
    '''
    robots = set()
    with gzip.open(filename) as f:
        for line in io.TextIOWrapper(f,encoding='ascii'):
            fields = line.split()
            if fields[6] == '/robots.txt':
                robots.add(fields[0])
    return robots


def find_all_robots(logdir):
    '''
    Find all hosts across and entire sequence of files
    '''
    files = glob.glob(logdir+'/*.log.gz')
    all_robots = set()
    for robots in map(find_robots, files): # map根据提供的函数对指定序列做映射
        all_robots.update(robots)
    return all_robots

if __name__ == '__main__':
    import time
    start = time.time()
    robots = find_all_robots("../data")
    end = time.time()
    for ipaddr in robots:
        print(ipaddr)
    print('Took {:f} seconds'.format(end-start))   